

######################
## PACKAGES
######################
library(dplyr)
library(haven)
library(Hmisc)
library(MASS)
library(modi)
library(readxl)

#######################################################
## STEP 1: Import covariance over time
#######################################################

covariances <- read_excel("NES_covariances_USA.xlsx")

#######################################################
## STEP 2: Make variance/covariance matrix 
#######################################################
d <- read_excel("Main dataset for United States, from Gilens 2012.xlsx")

d <- d %>% dplyr::filter(!is.na(PRED90_SW)&MALE_FAV!=0)

######## MIDDLE STOP ##########
d$pcFAV_SEX_MEN <- d$MALE_FAV/(d$MALE_FAV+d$MALE_OPP)
d$pcFAV_SEX_WOMEN <- d$FEMALE_FAV/(d$FEMALE_FAV+d$FEMALE_OPP)
d$S_SEX_MEN <- d$MALE_FAV+d$MALE_OPP
d$S_SEX_FEMALE <- d$FEMALE_FAV+d$FEMALE_OPP

# Make percentage variables HOUSEHOLD income 
d$pcFAV_INC1 <- d$INC1_FAV/(d$INC1_FAV+d$INC1_OPP)
d$pcFAV_INC2 <- d$INC2_FAV/(d$INC2_FAV+d$INC2_OPP)
d$pcFAV_INC3 <- d$INC3_FAV/(d$INC3_FAV+d$INC3_OPP)
d$pcFAV_INC4 <- d$INC4_FAV/(d$INC4_FAV+d$INC4_OPP)
d$pcFAV_INC5 <- d$INC5_FAV/(d$INC5_FAV+d$INC5_OPP)
d$pcFAV_INC6 <- d$INC6_FAV/(d$INC6_FAV+d$INC6_OPP)
d$pcFAV_INC7 <- d$INC7_FAV/(d$INC7_FAV+d$INC7_OPP)
d$pcFAV_INC8 <- d$INC8_FAV/(d$INC8_FAV+d$INC8_OPP)
d$pcFAV_INC9 <- d$INC9_FAV/(d$INC9_FAV+d$INC9_OPP)
d$pcFAV_INC10 <- d$INC10_FAV/(d$INC10_FAV+d$INC10_OPP)
d$pcFAV_INC11 <- d$INC11_FAV/(d$INC11_FAV+d$INC11_OPP)

# Set N for each income group to N who responded
d$S_HINC1 <- (d$INC1_FAV+d$INC1_OPP)
d$S_HINC2 <- (d$INC2_FAV+d$INC2_OPP)
d$S_HINC3 <- (d$INC3_FAV+d$INC3_OPP)
d$S_HINC4 <- (d$INC4_FAV+d$INC4_OPP)
d$S_HINC5 <- (d$INC5_FAV+d$INC5_OPP)
d$S_HINC6 <- (d$INC6_FAV+d$INC6_OPP)
d$S_HINC7 <- (d$INC7_FAV+d$INC7_OPP)
d$S_HINC8 <- (d$INC8_FAV+d$INC8_OPP)
d$S_HINC9 <- (d$INC9_FAV+d$INC9_OPP)
d$S_HINC10 <- (d$INC10_FAV+d$INC10_OPP)
d$S_HINC11 <- (d$INC11_FAV+d$INC11_OPP)

# OTHER 
d$ASKED_YEAR <- d$YEAR
d$ORG_DATASET_N <- d$MALE_FAV+d$MALE_OPP+d$MALE_DK+d$FEMALE_FAV+d$FEMALE_OPP+d$FEMALE_DK

d$OVERALL_FAV <- d$MALE_FAV+d$FEMALE_FAV
d$OVERALL_OPP <- d$MALE_OPP+d$FEMALE_OPP


d$pred_i90_men <- NA
d$pred_i50_men <- NA
d$pred_i10_men <- NA
d$pred_i90_women <- NA
d$pred_i50_women <- NA
d$pred_i10_women <- NA

for(i in 1:nrow(d)){
  drow <- d[i,]
  # INCOME/SUPPORT COVARIANCE
  d1 <- drow %>% dplyr::select(starts_with("S_HINC"))
  d1 <- as.data.frame(t(d1))
  d1$prop <- d1$V1/sum(d1$V1, na.rm=T)
  d1$cumu <- cumsum(d1$prop)
  d1 <- mutate(d1, score = ((cumu - lag(cumu))/2)+lag(cumu)) # Make percentile midpoint scores
  d1$score[1] <- d1$cumu[1]-(d1$cumu[1]/2) # Make percentile midpoint score for first category since formula above returns NA for that one
  d1$income <- d1$Var1
  d1 <- add_rownames(d1, "income")
  d1 <- add_rownames(d1, "income")
  names(d1)[names(d1)=="V1"] <- "n"
  d1 <- dplyr::select(d1, income, score, n)
  d2 <- drow %>% dplyr::select(starts_with("pcFAV_INC"))
  d2 <- as.data.frame(t(d2)) 
  d2$income <- 1:nrow(d2) 
  names(d2)[names(d2)=="V1"] <- "sup"
  d3 <- merge(d1,d2, by="income")
  cor(d3$sup,d3$score,use="complete.obs")
  d3<-na.omit(d3)
  d4<- d3 %>% dplyr::select(score,sup)
  sup_inc_cov<-cov.wt(d4,wt=d3$n)$cov[1,2]
  inc_var<-cov.wt(d4,wt=d3$n)$cov[1,1]
  inc_mean <- weighted.mean(d3$score,w=d3$n)
  
  # SEX/SUPPORT COVARIANCE
  d1 <- drow %>% dplyr::select(starts_with("S_SEX"))
  d1 <- as.data.frame(t(d1))
  d1$sex <- rownames(d1)
  d1$sex <- ifelse(d1$sex=="S_SEX_MEN",1,0)
  names(d1)[names(d1)=="V1"] <- "n"
  d1 <- dplyr::select(d1, sex, n)
  d2 <- drow %>% dplyr::select(starts_with("pcFAV_SEX"))
  d2 <- as.data.frame(t(d2)) 
  d2$sex <- rownames(d2)
  d2$sex <- ifelse(d2$sex=="pcFAV_SEX_MEN",1,0)
  names(d2)[names(d2)=="V1"] <- "sup"
  d3 <- merge(d1,d2, by="sex")
  cor(d3$sup,d3$sex,use="complete.obs")
  d3<-na.omit(d3)
  d3$prop <- d3$n/sum(d3$n, na.rm=T)
  d4<- d3 %>% dplyr::select(sex,sup)
  sup_sex_cov<-cov.wt(d4,wt=d3$n,method="ML")$cov[1,2]
  sex_var<-cov.wt(d4,wt=d3$n,method="ML")$cov[1,1] # NB: Variance for "sup" NOT CORRECT (Calculated with correct weights below)
  sex_mean <- weighted.mean(d3$sex,w=d3$n)
  # SUPPORT VARIANCE
  d1 <- drow %>% dplyr::select(OVERALL_FAV,OVERALL_OPP)
  d1 <- as.data.frame(t(d1))
  d1$sup<-c(1,0)
  sup_var <- wtd.var(d1$sup,d1$V1) 
  sup_mean <- weighted.mean(d1$sup,w=d1$V1)
  # BUILD VARIANCE COVARIANCE MATRIX
  sup <- c(sup_var,sup_inc_cov,sup_sex_cov)
  inc <- c(sup_inc_cov,inc_var,covariances$cov[covariances$year==drow$ASKED_YEAR])
  sex <- c(sup_sex_cov,covariances$cov[covariances$year==drow$ASKED_YEAR],sex_var)
  varcovar <- data.frame(sup,inc,sex)
  rownames(varcovar) <- c("sup","inc","sex")
  varcovar <- data.matrix(varcovar)
  varcovar # VARIANCE COVARIANCE MATRIX
  means <- c(sup_mean,inc_mean,sex_mean)
  data<-as.data.frame(mvrnorm(n=round(drow$ORG_DATASET_N,0),means,varcovar,empirical=T)) # Generate dataset based on this var-covar-matrix
  m1 <- glm(sup ~ inc + I(inc^2) + sex, data=data) # Income, income^2, sexcation, and sexcation^2 as IVs, just like Gilens (2012, 93-94)
  d4 <- expand.grid(inc=c(0.90,0.50,0.10),sex=c(1,0))
  d4$pred <- predict(m1, d4, type="response")
  d$pred_i90_men[i] <- d4$pred[1]
  d$pred_i50_men[i] <- d4$pred[2]
  d$pred_i10_men[i] <- d4$pred[3]
  d$pred_i90_women[i] <- d4$pred[4]
  d$pred_i50_women[i] <- d4$pred[5]
  d$pred_i10_women[i] <- d4$pred[6]
}

############################################################################
############################## END #########################################
############################################################################


write.xlsx(d, "Main dataset for United States, from Gilens 2012.xlsx") # Writes out master dataset with the new variables added



